#!/usr/bin/env python
#-*-coding:utf-8-*-


PAD_STR = "<PAD>"
OOV_STR = "<OOV>"


def pre_char_vocab(pre_embeding_path):
    f = open(pre_embeding_path, 'r', errors="ignore")
    outf = open("../../data/word_vocab.txt", 'w')

    m, n = f.readline().split()
    n = int(n)
    m = int(m)
    char_to_id = {}
    char_to_id[PAD_STR] = 0
    char_to_id[OOV_STR] = 1

    s_count = 0
    for i, line in enumerate(f):
        line = line.strip("\n").strip()
        line = line.split()
        if len(line) != n + 1:
            s_count += 1
            word = "</s>"
        else:
            word = line[0]
        if word not in char_to_id:
            char_to_id[word] = len(char_to_id)

    for k, v in char_to_id.items():
        outf.write(" ".join([k, str(v)]) + "\n")
    print(s_count)


pre_embeding_path = "../../data/word_vec_100.txt"
pre_char_vocab(pre_embeding_path)
